#!/bin/bash
#
# Copyright (c) 2017 MariaDB Corporation Ab
#
# Use of this software is governed by the Business Source License included
# in the LICENSE.TXT file.
#
# Change Date: 2021-12-01
#
# On the date above, in accordance with the Business Source License, use
# of this software will be governed by version 2 or later of the General
# Public License.
#
VERBOSE=false
QUIET=false
COMPRESS=false
INSTALL_DIR=/usr/local/mariadb/columnstore
PM1=none
BACKUP_SERVER_LOCATION=none
DRY_RUN=false
REMOTE_USER=root
NUMBER_BACKUPS=3
NUMBER_CONCURRENT=5
CONFIG_FILE=false
LOG_FILE=$0.log

exec 3>&1 1>>${LOG_FILE} 2>&1

startTime=`date +%s`

printLog () {
    LOGTIME="[`date "+%Y-%m-%d %H:%M:%S"`] "
    echo $LOGTIME$* | tee /dev/fd/3
}

verbosePrint () {
    if [ "$VERBOSE" = true ]; then
        printLog $*
    fi
}

##
## ERROR CODES
##
## 1   - command line parameter or config file issue detected
## 2   - missing rsync or xmllint
## 3   - detected issue with disk space
## 4   - detected bad configuration file settings
## 5   - rsync command failed with an error
## 255 - could not connect via passwordless ssh

reportStatus () {
    if [ $1 != 0 ]; then
        printLog "Backup Status: Failed"
    else
        printLog "Backup Status: Success"
    fi
    endTime=`date +%s`
    runtime=$(( $endTime - $startTime ))
    runtimeHours=$(( $runtime / 3600 ))
    runtimeMinutes=$(( $(( $runtime % 3600 )) / 60 ))
    runtimeSeconds=$(( $runtime % 60 ))
    line=$(printf "%02d:%02d:%02d" $runtimeHours $runtimeMinutes $runtimeSeconds)
    printLog Runtime: $line
    exit $1
}

printLog "[$USER] $0 $@"

if [ -e columnstoreBackup.config ]; then
    source columnstoreBackup.config
fi
REcheck='^[0-9]+$'
if ! [[ $NUMBER_BACKUPS =~ $REcheck ]] ; then
   printLog "ERROR: NUMBER_BACKUPS not a number" >&2
   reportStatus 1
fi
if [ $NUMBER_BACKUPS -lt 0 ] || [ $NUMBER_BACKUPS -gt 20 ]; then
    printLog "ERROR: NUMBER_BACKUPS must be an integer between 1 and 20 (value: $NUMBER_BACKUPS)" >&2
    reportStatus 1
fi

###
# Print Fucntions
###

helpPrint () {
          ################################################################################
    echo "" 1>&3
    echo "MariaDB ColumnStore Automated Backup Tool" 1>&3
    echo "" 1>&3
    echo "This tool is meant to automate the ColumnStore backup procedure documented at:" 1>&3
    echo "" 1>&3
    echo "https://mariadb.com/kb/en/mariadb/columnstore-backup-and-recovery/" 1>&3
    echo "" 1>&3
    echo "The tool is designed to be run on the system storing the backups. It requires" 1>&3
    echo "the backup system to have passwordless login enabled for the account that" 1>&3
    echo "MariaDB ColumnStore was installed and maintained. (Assumes root by default)" 1>&3
    echo "" 1>&3
    echo "This tool must be run as root or with sudo." 1>&3
    echo "" 1>&3
    echo "The default behavior is to run rsync locally to each UM and PM." 1>&3
    echo "PM data is stored in data directories labeled pm[moduleID]dbroot[DBrootID]" 1>&3
    echo "UM data is stored in directories labeled um[moduleID]" 1>&3
    echo "my.cnf data is stored in directory cnf" 1>&3
    echo "Columnstore.xml is stored in the top level of backupServerLocation" 1>&3
    echo "" 1>&3
    echo "The -n option uses the rsync link-dest option to enable incremental backups." 1>&3
    echo "These are stored in backup.1 thru backup.[n-1] from newest to oldest." 1>&3
    echo "The impact on extra space needed is based on the amount data that changes." 1>&3
    echo "with little to no data set changes the incremental backup size is small." 1>&3
    echo "" 1>&3
    echo "Usage: $0 [options] activeParentOAM backupServerLocation" 1>&3
    echo "" 1>&3
    echo "activeParentOAM           IP address of ColumnStore server" 1>&3
    echo "                             (Active parent OAM module on multi-node install)" 1>&3
    echo "backupServerLocation      Path to the directory for storing backup files." 1>&3
    echo "" 1>&3
    echo "OPTIONS:" 1>&3
    echo "-h,--help         Print this message and exit." 1>&3
    echo "-v,--verbose      Print more verbose execution details." 1>&3
    echo "-d,--dry-run      Dry run and executes rsync dry run with stats." 1>&3
    echo "-z,--compress     Utilize the compression option for rsync." 1>&3
    echo "-n [value]        Maximum number parallel rsync commands. (Default: 5)." 1>&3
    echo "--user=[user]     Change the user performing remote sessions. (Default: root)" 1>&3
    echo "" 1>&3
    echo "--install-dir=[PATH]  Change the install directory of ColumnStore." 1>&3
    echo "                          Default: /usr/local/mariadb/columnstore" 1>&3
    echo "" 1>&3
}

# Parse command line options.
while getopts hdn:zv-: OPT; do
    case "$OPT" in
        h)
            printLog $USAGE
            helpPrint
            exit 0
            ;;
        d)
            DRY_RUN=true
            ;;
        n)
            NUMBER_CONCURRENT=$OPTARG
            if ! [[ $NUMBER_CONCURRENT =~ $REcheck ]] ; then
               printLog "ERROR: NUMBER_CONCURRENT not a number" >&2
               reportStatus 1
            fi
            if [ $NUMBER_CONCURRENT -lt 0 ] || [ $NUMBER_CONCURRENT -gt 100 ]; then
                printLog "ERROR: NUMBER_CONCURRENT must be an integer between 1 and 100 (value: $NUMBER_CONCURRENT)" >&2
                reportStatus 1
            fi
            ;;
        z)
            COMPRESS=true
            ;;      
        v)
            VERBOSE=true
            ;;         
        -)  LONG_OPTARG="${OPTARG#*=}"
            ## Parsing hack for the long style of arguments.
            case $OPTARG in
                help )  
                    helpPrint
                    exit 0
                    ;;            
                compress )  
                    COMPRESS=true 
                    ;;
                dry-run )
                    DRY_RUN=true
                    ;;   
                verbose)
                    VERBOSE=true
                    ;;                    
                install-dir=?* )  
                    INSTALL_DIR="$LONG_OPTARG" 
                    ;;
                user=?* )  
                    REMOTE_USER="$LONG_OPTARG" 
                    ;;                    
                install-dir* )  
                    printLog "No arg for --$OPTARG option" >&2
                    reportStatus 1
                    ;;
                user* )  
                    printLog "No arg for --$OPTARG option" >&2
                    reportStatus 1
                    ;;
                help* )  
                    helpPrint
                    exit 0
                    ;;                                        
                compress* )
                    printLog "No arg allowed for --$OPTARG option" >&2
                    reportStatus 1 
                    ;;
                dry-run* )
                    printLog "No arg allowed for --$OPTARG option" >&2
                    reportStatus 1
                    ;;      
                verbose* )
                    printLog "No arg allowed for --$OPTARG option" >&2
                    reportStatus 1
                    ;;                    
                '' )
                    break ;; # "--" terminates argument processing
                * )
                    printLog "Illegal option --$OPTARG" >&2
                    reportStatus 1
                    ;;
            esac 
            ;;       
        \?)
            # getopts issues an error message
            printLog $USAGE >&2
            reportStatus 1
            ;;
    esac
done

# Remove the switches we parsed above.
shift `expr $OPTIND - 1`

# We want 2 non-option argument. 
if [ $# -ne 2 ]; then
    if [ $# -lt 2 ]; then
        printLog "Missing arguments." >&2
    fi
    if [ $# -gt 2 ]; then
        printLog "Unknown extra arguments." >&2
    fi
    printLog $USAGE >&2
    reportStatus 1
fi

#
PM1=$1
BACKUP_SERVER_LOCATION=$2

## Check if root or sudo
if [[ $EUID -ne 0 ]]; then
   printLog "ERROR: This script must be run as root or with sudo" 
   reportStatus 1
fi

## Check rsync and xmllint are available
if ! type rsync > /dev/null; then
    printLog "rsync is not installed. Please install and rerun." >&2
    reportStatus 2
fi
if ! type xmllint > /dev/null; then
    printLog "xmllint is not installed. Please install and rerun." >&2
    reportStatus 2
fi

if [ ! -d "$BACKUP_SERVER_LOCATION" ]; then
    printLog "Directory $BACKUP_SERVER_LOCATION does not exist." >&2
    reportStatus 1
fi

## setup rsync option lists
rsyncCommand="rsync "
rsyncOptions="-a "
rsyncLongOptions="--delete "
if [ "$COMPRESS" = true ]; then
    rsyncOptions=$rsyncOptions"-z " 
fi
if [ "$VERBOSE" = true ]; then
    rsyncOptions=$rsyncOptions""
    rsyncLongOptions=$rsyncLongOptions""
fi

rsyncCommand="$rsyncCommand$rsyncOptions$rsyncLongOptions"

if ! ssh -q -o "BatchMode yes" $REMOTE_USER@$PM1 ls $INSTALL_DIR > /dev/null; then
    printLog "ERROR: Cannot connect to $REMOTE_USER@$PM1" >&2
    printLog "Check that ssh and passwordless login are available." >&2
    reportStatus 255
fi

## EX ARG pm1dbroot1
rotateBackups () {
for (( backupNumber=1; backupNumber<=$NUMBER_BACKUPS; backupNumber++ ))
do
    if [ ! -e $BACKUP_SERVER_LOCATION/backup.$backupNumber ]; then
        mkdir $BACKUP_SERVER_LOCATION/backup.$backupNumber
    fi
done

for (( backupNumber=$NUMBER_BACKUPS; backupNumber>=1; backupNumber-- ))
do
    if [ $backupNumber -eq $NUMBER_BACKUPS ]; then
        rm -rf $BACKUP_SERVER_LOCATION/backup.$backupNumber
    else
        mv $BACKUP_SERVER_LOCATION/backup.$backupNumber $BACKUP_SERVER_LOCATION/backup.$(($backupNumber+1))
    fi
done
}

moveBackupFiles () {
moveThis=$1
destPath=$2
if [ ! -e $BACKUP_SERVER_LOCATION/backup.1 ]; then
    mkdir $BACKUP_SERVER_LOCATION/backup.1
fi
if [ "$DRY_RUN" = false ]; then
    if [ -e $BACKUP_SERVER_LOCATION/$moveThis ]; then
        mv $BACKUP_SERVER_LOCATION/$moveThis $BACKUP_SERVER_LOCATION/$destPath
    fi
fi
}

executeRsync () {
dryRunFlag=$1
extraOptions=$2
source=$3
dest=$4

## setup rsync option lists
if [ "$dryRunFlag" = true ]; then
    extraOptions="-n --stats "$extraOptions
fi

command="$rsyncCommand $extraOptions $source $dest"


if [ "$dryRunFlag" = false ]; then
    verbosePrint "$command"
    if ! $command; then
        printLog "ERROR: Rsync command failed." >&2
        printLog "ERROR Command: $command" >&2
        reportStatus 5
    fi
else
    printLog "$command"
    $command    
fi
}

executeRsyncBackground () {
dryRunFlag=$1
resultsDir=$2
extraOptions=$3
source=$4
dest=$5

## setup rsync option lists
if [ "$dryRunFlag" = true ]; then
    extraOptions="-n --stats "$extraOptions
fi

command="$rsyncCommand $extraOptions $source $dest"

if [ "$dryRunFlag" = false ]; then
    verbosePrint "$command"
    { $command ; echo "$?" > "$resultsDir" ; } &
else
    printLog "$command"
    $command
fi
}

waitRsyncBackground () {
dataSize=$1
dir=$2
activeBackups=$3
if [ "$DRY_RUN" = false ]; then
    printLog "Waiting for backups to complete..."
    monitorProgress $dataSize $dir $activeBackups
    wait
    for file in "$dir"/*; do
        if [ $(<"$file") != 0 ]; then
            printLog "ERROR: RSYNC failed $file with Error: $(<"$file")" >&2
            printLog "Backup may not be usable." >&2
            rm -r "$dir"                       
            reportStatus 5
        fi
    done
    rm -r "$dir"                       
fi
}

###############################################################################
# Get the system info from provided PM1
###############################################################################
getSystemInfo () {
# Go grab PM1's columnstore.xml file and parse it to find out information on system nodes.
moveBackupFiles Columnstore.xml ./backup.1/
executeRsync false --link-dest=./backup.1 $REMOTE_USER@$PM1:$INSTALL_DIR/etc/Columnstore.xml $BACKUP_SERVER_LOCATION 

moveBackupFiles releasenum ./backup.1/
executeRsync false --link-dest=./backup.1 $REMOTE_USER@$PM1:$INSTALL_DIR/releasenum $BACKUP_SERVER_LOCATION

## parse it
systemName=$(xmllint --xpath 'string(//SystemName)' $BACKUP_SERVER_LOCATION/Columnstore.xml)
singleServerInstall=$(xmllint --xpath 'string(//SingleServerInstall)' $BACKUP_SERVER_LOCATION/Columnstore.xml)
serverTypeInstall=$(xmllint --xpath 'string(//ServerTypeInstall)' $BACKUP_SERVER_LOCATION/Columnstore.xml)
PMwithUM=$(xmllint --xpath 'string(//PMwithUM)' $BACKUP_SERVER_LOCATION/Columnstore.xml)
DBRootStorageType=$(xmllint --xpath 'string(//DBRootStorageType)' $BACKUP_SERVER_LOCATION/Columnstore.xml)

umModuleCount=$(xmllint --xpath "string(//ModuleCount2)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
pmModuleCount=$(xmllint --xpath "string(//ModuleCount3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)

DBRootCount=$(xmllint --xpath 'string(//DBRootCount)' $BACKUP_SERVER_LOCATION/Columnstore.xml)
for (( dbRootID=1; dbRootID<=$DBRootCount; dbRootID++ ))
do
    DBRoot[$dbRootID]=$(xmllint --xpath "string(//DBRoot$dbRootID)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
done

if [ $singleServerInstall == "n" ]; then
    detectError=false
    for (( moduleID=1; moduleID<=$umModuleCount; moduleID++ ))
    do
        umModuleIP1=$(xmllint --xpath "string(//ModuleIPAddr$moduleID-1-2)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        umModuleIP2=$(xmllint --xpath "string(//ModuleIPAddr$moduleID-2-2)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        umModuleHostname1=$(xmllint --xpath "string(//ModuleHostName$moduleID-1-2)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        umModuleHostname2=$(xmllint --xpath "string(//ModuleHostName$moduleID-2-2)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        if ssh -q -o "BatchMode yes" $REMOTE_USER@$umModuleIP1 exit; then
            umModuleIP[$moduleID]=$umModuleIP1
            umModuleHostname[$moduleID]=$umModuleHostname1
        elif ssh -q -o "BatchMode yes" $REMOTE_USER@$umModuleIP2 exit; then
            umModuleIP[$moduleID]=$umModuleIP2
            umModuleHostname[$moduleID]=$umModuleHostname2
        else
            printLog "ERROR: cannot Connect to UM$moduleID" >&2
            printLog "(IP1 = $umModuleIP1)" >&2
            printLog "(IP2 = $umModuleIP2)" >&2
            detectError=true
        fi
    done

    for (( moduleID=1; moduleID<=$pmModuleCount; moduleID++ ))
    do
        pmModuleIP1=$(xmllint --xpath "string(//ModuleIPAddr$moduleID-1-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        pmModuleIP2=$(xmllint --xpath "string(//ModuleIPAddr$moduleID-2-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        pmModuleHostname1=$(xmllint --xpath "string(//ModuleHostName$moduleID-1-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        pmModuleHostname2=$(xmllint --xpath "string(//ModuleHostName$moduleID-2-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        
        if ssh -q -o "BatchMode yes" $REMOTE_USER@$pmModuleIP1 exit; then
            pmModuleIP[$moduleID]=$pmModuleIP1
            pmModuleHostname[$moduleID]=$pmModuleHostname1
        elif ssh -q -o "BatchMode yes" $REMOTE_USER@$pmModuleIP2 exit; then
            pmModuleIP[$moduleID]=$pmModuleIP2
            pmModuleHostname[$moduleID]=$pmModuleHostname2
        else
            printLog "ERROR: Cannot connect to PM$moduleID" >&2
            printLog "(IP1 = $pmModuleIP1)" >&2
            printLog "(IP2 = $pmModuleIP2)" >&2
            detectError=true
        fi
        
        pmModuleDBRootCount[$moduleID]=$(xmllint --xpath "string(//ModuleDBRootCount$moduleID-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)     
    done
    
    if [ $serverTypeInstall == "2" ]; then
        umModuleCount=$pmModuleCount
        for (( moduleID=1; moduleID<=$umModuleCount; moduleID++ ))
        do
            umModuleIP[$moduleID]=${pmModuleIP[$moduleID]}
            umModuleHostname[$moduleID]=${pmModuleHostname[$moduleID]}     
        done
    fi
    if [ "$detectError" = true ]; then
        printLog "Check that ssh and passwordless login are available." >&2
        reportStatus 255 
    fi
    
elif [ $singleServerInstall == "y" ]; then
    
    for (( moduleID=1; moduleID<=$pmModuleCount; moduleID++ ))
    do
        pmModuleDBRootCount[$moduleID]=$(xmllint --xpath "string(//ModuleDBRootCount$moduleID-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)      
    done
    umModuleCount=1
    umModuleIP[1]=$PM1
    pmModuleIP[1]=$PM1
       
else
    printLog "Unknown install type = $singleServerInstall" >&2
    reportStatus 4
fi

}

measureDiskSpace () {
    for (( moduleID=1; moduleID<=$pmModuleCount; moduleID++ ))
    do
        for (( moduleDBRootID=1; moduleDBRootID<=${pmModuleDBRootCount[$moduleID]}; moduleDBRootID++ ))
        do
            thisDBrootID=$(xmllint --xpath "string(//ModuleDBRootID$moduleID-$moduleDBRootID-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
            command="ssh $REMOTE_USER@${pmModuleIP[$moduleID]} du -s -b ${DBRoot[$thisDBrootID]}"
            verbosePrint "$command"
            commandReturn=$($command)
            returnValues=($commandReturn)
            if [ ${returnValues[1]} == ${DBRoot[$thisDBrootID]} ]; then
                totalDiskSpaceNeeded=$(( $totalDiskSpaceNeeded + ${returnValues[0]} ))
                totalDiskSpaceNeededPM=$(( $totalDiskSpaceNeededPM + ${returnValues[0]} ))                
            else
                printLog "failed return from command: $command" >&2
                reportStatus 3
            fi
            if [ $PMwithUM == "y" ]; then
                command="ssh $REMOTE_USER@${pmModuleIP[$moduleID]} du -s -b $INSTALL_DIR/mysql/db"
                verbosePrint "$command"
                commandReturn=$($command)
                returnValues=($commandReturn)
                if [ ${returnValues[1]} == $INSTALL_DIR/mysql/db ]; then
                    totalDiskSpaceNeeded=$(( $totalDiskSpaceNeeded + ${returnValues[0]} ))
                    totalDiskSpaceNeededPM=$(( $totalDiskSpaceNeededPM + ${returnValues[0]} ))                
                else
                    printLog "failed return from command: $command" >&2
                    reportStatus 3
                fi                
            fi
        done        
    done
    for (( moduleID=1; moduleID<=$umModuleCount; moduleID++ ))
    do
        command="ssh $REMOTE_USER@${umModuleIP[$moduleID]} du -s -b $INSTALL_DIR/mysql/db"
        commandReturn=$($command)
        returnValues=($commandReturn)
        if [ ${returnValues[1]} == $INSTALL_DIR/mysql/db ]; then
            totalDiskSpaceNeeded=$(( $totalDiskSpaceNeeded + ${returnValues[0]} ))
            totalDiskSpaceNeededUM=${returnValues[0]}               
        else
            printLog "failed return from command: $command" >&2
            reportStatus 3
        fi
    done
    totalDiskSpaceAvailable=$(df -B1 $BACKUP_SERVER_LOCATION | awk 'NR==2 {print $4}')
}

###############################################################################
# monitorProgress
###############################################################################
monitorProgress () {
sizeOfData=$1
resultDir=$2
activeBackups=$3
percentDone=0
oldSizeCopied=0
totalSizeCopied=0
printSeconds=0
printMinutes=0
printHours=0
sleepInt=5
loopCount=0
windowSize=5
stallCount=0
rsyncStalled=false
if (( $sizeOfData <= 0 )); then
    printLog "WARNING: Rsync data size reported as $sizeOfData" >&2
    percentDone=100
fi
for (( winNum=0; winNum<$windowSize; winNum++ ))
do
    transferDiffs[$winNum]=0
done
while [ $percentDone -lt 99 ]
do
    cmdRtn=$(du -s -b $BACKUP_SERVER_LOCATION 2>/dev/null)
    returnValues=($cmdRtn)
    if [ ${returnValues[1]} == $BACKUP_SERVER_LOCATION ]; then
        oldSizeCopied=$totalSizeCopied
        totalSizeCopied=${returnValues[0]}
        if (( $oldSizeCopied == 0 )); then
            oldSizeCopied=$totalSizeCopied
        fi
    fi
    # Compute the percentage.
    percentDone=$(( $(( $totalSizeCopied * 100 )) / $sizeOfData ))
    # Compute the number of blocks to represent the percentage.
    if [ $percentDone -gt 99 ]; then
        break
    fi
    numberSymbols=$(( percentDone / 4 ))
    # Create the progress bar string.
    bar="Progress ["
    for (( num=0; num<=25; num++ ))
    do
        if (( $num < $numberSymbols )); then
            bar=$bar"="
        elif (( $num == $numberSymbols )); then
            bar=$bar">"
        else
            bar=$bar" "
        fi
    done

    # Print the progress bar.
    diff=$(( $totalSizeCopied - $oldSizeCopied )) 

    if (( $diff > 0 )); then   
        remainingCopy=$(( $sizeOfData - $totalSizeCopied ))
    else
        stallCount=$(( $stallCount + 1 ))
    fi
    
    winNum=$(( $loopCount % $windowSize ))
    transferDiffs[$winNum]=$diff
    totalDiffs=0
    for (( winNum=0; winNum<$windowSize; winNum++ ))
    do
        totalDiffs=$(($totalDiffs+${transferDiffs[$winNum]}))
    done
    averageDiffs=$(($totalDiffs / $windowSize))
    
    if (( $averageDiffs > 1000000 )); then
        rate=$(( $averageDiffs / 1000000 / $sleepInt ))" Mbps "
    elif (( $averageDiffs > 1000 )); then
        rate=$(( $averageDiffs / 1000 / $sleepInt ))" kbps "
    else
        rate=$averageDiffs" bits/second "
    fi
    
    if (( $averageDiffs > $sleepInt )); then   
        remainingSeconds=$(( $remainingCopy / $(( $averageDiffs / $sleepInt )) ))
        printHours=$(( $remainingSeconds / 3600 ))
        printMinutes=$(( $(( $remainingSeconds % 3600 )) / 60 ))
        printSeconds=$(( $remainingSeconds % 60 ))
    else
        printSeconds=0
        printMinutes=0
        printHours=0
    fi
    
    bar=$bar"] ($percentDone%) $rate "
    line=$(printf "%02d:%02d:%02d %s" $printHours $printMinutes $printSeconds "Approx Time Remaining          ")
    echo -en "${bar}${line}\r" 1>&3
    if [ $stallCount -gt 5 ]; then
        rsyncStalled=true;
        break
    fi
    completedBackups=$(ls -Anq $resultDir | grep -c '^-')
    if [ $completedBackups -eq $activeBackups ]; then
        echo -en "${bar}${line}\n" 1>&3    
        break
    fi
    sleep $sleepInt
    loopCount=$(($loopCount + 1))
done

if [ $rsyncStalled == false ] && [ $percentDone -ge 99 ]; then
    percentDone=100
    bar="Progress [=========================>] ($percentDone%) $rate "
    line=$(printf "%02d:%02d:%02d %s" $printHours $printMinutes $printSeconds "Approx Time Remaining          ")
    echo -en "${bar}${line}\n" 1>&3
fi
}

###############################################################################
# Backup PM
###############################################################################
backupPMs () {
printLog "Backing up PMs"
activeBackups=0
dir=$(mktemp -d)
## loop the pmModules
for (( moduleID=1; moduleID<=$pmModuleCount; moduleID++ ))
do
    for (( moduleDBRootID=1; moduleDBRootID<=${pmModuleDBRootCount[$moduleID]}; moduleDBRootID++ ))
    do
        thisDBrootID=$(xmllint --xpath "string(//ModuleDBRootID$moduleID-$moduleDBRootID-3)" $BACKUP_SERVER_LOCATION/Columnstore.xml)
        pmBackupDir="pm""$moduleID""dbroot""$moduleDBRootID"
        if [ ! -e $BACKUP_SERVER_LOCATION/$pmBackupDir ]; then
            mkdir $BACKUP_SERVER_LOCATION/$pmBackupDir
        fi
        printLog "Backing up $pmBackupDir"
        moveBackupFiles $pmBackupDir ./backup.1/
        # Launch it in background
        executeRsyncBackground $DRY_RUN /$dir/$pmBackupDir "--copy-links --link-dest=../backup.1/$pmBackupDir" $REMOTE_USER@${pmModuleIP[$moduleID]}:${DBRoot[$thisDBrootID]} $BACKUP_SERVER_LOCATION/$pmBackupDir
        ((activeBackups++))
        
        if [ $PMwithUM == "y" ]; then
            pmBackupDirDB="pm"$moduleID"DB"
            printLog "Backing up $pmBackupDir local query data"
            if [ ! -e $BACKUP_SERVER_LOCATION/$pmBackupDirDB ]; then
                mkdir $BACKUP_SERVER_LOCATION/$pmBackupDirDB
            fi
            executeRsync $DRY_RUN --link-dest=../backup.1/$pmBackupDirDB $REMOTE_USER@${pmModuleIP[$moduleID]}:$INSTALL_DIR/mysql/db $BACKUP_SERVER_LOCATION/$pmBackupDirDB           
        fi
        if [ $activeBackups -ge $NUMBER_CONCURRENT ]; then
            waitRsyncBackground $totalDiskSpaceNeededPM $dir $activeBackups
            dir=$(mktemp -d)
            activeBackups=0
        fi
    done        
done
if [ $activeBackups -gt 0 ]; then
    waitRsyncBackground $totalDiskSpaceNeededPM $dir $activeBackups
fi
printLog "DONE"
}

###############################################################################
# Backup UM
###############################################################################
backupUM () {
printLog "Backing up UM"
## dir=$(mktemp -d)
## loop the pmModules
for (( moduleID=1; moduleID<=$umModuleCount; moduleID++ ))
do
    umBackupDir="um""$moduleID"
    if [ ! -e $BACKUP_SERVER_LOCATION/$umBackupDir ]; then
        mkdir $BACKUP_SERVER_LOCATION/$umBackupDir
    fi
    
    printLog "Backing up $umBackupDir"
    moveBackupFiles $umBackupDir ./backup.1/
    # Launch it in background
    executeRsync $DRY_RUN --link-dest=../backup.1/$umBackupDir $REMOTE_USER@${umModuleIP[$moduleID]}:$INSTALL_DIR/mysql/db $BACKUP_SERVER_LOCATION/$umBackupDir
done

#waitRsyncBackground $totalDiskSpaceNeededUM $dir
printLog "DONE"
}


backupCnf () {
backupCnfDir="cnf"
if [ ! -e $BACKUP_SERVER_LOCATION/$backupCnfDir ]; then
    mkdir $BACKUP_SERVER_LOCATION/$backupCnfDir
fi
moveBackupFiles $backupCnfDir ./backup.1/

executeRsync $DRY_RUN --link-dest=../backup.1/$backupCnfDir $REMOTE_USER@$PM1:$INSTALL_DIR/mysql/my.cnf $BACKUP_SERVER_LOCATION/$backupCnfDir/
executeRsync $DRY_RUN --link-dest=../backup.1/$backupCnfDir $REMOTE_USER@$PM1:$INSTALL_DIR/mysql/my.cnf.d $BACKUP_SERVER_LOCATION/$backupCnfDir/
}


###############################################################################
# suspendDBWrites
###############################################################################
suspendDBWrites () {
if [ "$DRY_RUN" = false ]; then
    if [ "$VERBOSE" = true ]; then
        ssh $REMOTE_USER@$PM1 $INSTALL_DIR/bin/mcsadmin suspendDatabaseWrites y
    else
        printLog "Suspend DB Writes"
        ssh $REMOTE_USER@$PM1 $INSTALL_DIR/bin/mcsadmin suspendDatabaseWrites y > /dev/null
    fi
fi
}


###############################################################################
# resumeDBWrites
###############################################################################
resumeDBWrites () {
if [ "$DRY_RUN" = false ]; then
    if [ "$VERBOSE" = true ]; then
        ssh $REMOTE_USER@$PM1 $INSTALL_DIR/bin/mcsadmin resumeDatabaseWrites y
    else
        printLog "Resume DB Writes"
        ssh $REMOTE_USER@$PM1 $INSTALL_DIR/bin/mcsadmin resumeDatabaseWrites y > /dev/null
    fi
fi    
}

###############################################################################
# Main Execution
###############################################################################
main () {

if [ "$DRY_RUN" = false ]; then
    rotateBackups
fi

## Get the system info from xml file
getSystemInfo

## Debug Prints
verbosePrint ""
verbosePrint "Configuration:"
verbosePrint ""
verbosePrint "Server PM1 address = $PM1"
verbosePrint "Backup Server location = $BACKUP_SERVER_LOCATION"
verbosePrint "SystemName: $systemName"
verbosePrint "SingleServerInstall: $singleServerInstall"
verbosePrint "serverTypeInstall:  $serverTypeInstall"
verbosePrint "PMwithUM:  $PMwithUM"
verbosePrint "DBRootStorageType: $DBRootStorageType"
verbosePrint ""
verbosePrint "UMs: $umModuleCount"
verbosePrint "PMs: $pmModuleCount"
for (( moduleID=1; moduleID<=$umModuleCount; moduleID++ ))
do
    verbosePrint "um$moduleID ip ${umModuleIP[$moduleID]}"
    verbosePrint "um$moduleID host ${umModuleHostname[$moduleID]}"
    verbosePrint ""
done

for (( moduleID=1; moduleID<=$pmModuleCount; moduleID++ ))
do
    verbosePrint "pm$moduleID ip ${pmModuleIP[$moduleID]}"
    verbosePrint "pm$moduleID host ${pmModuleHostname[$moduleID]}"
    verbosePrint "pm$moduleID DBRootCount: ${pmModuleDBRootCount[$moduleID]}"
    verbosePrint ""
done

verbosePrint "DBRootCount: $DBRootCount"
for (( dbRootID=1; dbRootID<=$DBRootCount; dbRootID++ ))
do
    verbosePrint "DBRoot$dbRootID: ${DBRoot[$dbRootID]}"
done

## Check if xmllint is available for use


## Measure disk space usage and compare to available
measureDiskSpace

#verbosePrint "totalDiskSpaceNeededPM:    $totalDiskSpaceNeededPM"
#verbosePrint "totalDiskSpaceNeededUM:    $totalDiskSpaceNeededUM"
#verbosePrint "totalDiskSpaceAvailable:   $totalDiskSpaceAvailable"

if (( $totalDiskSpaceAvailable < $totalDiskSpaceNeeded )); then
    printLog "ERROR: Not enough space on backup system" >&2
    printLog "(Needs = $totalDiskSpaceNeeded)" >&2
    printLog "(Avail = $totalDiskSpaceAvailable)" >&2
    reportStatus 3
fi

## suspend db writes
suspendDBWrites

backupCnf

backupPMs

backupUM

## resume db writes
resumeDBWrites

reportStatus 0
}

###############################################################################
# RUN MAIN
###############################################################################
main

reportStatus 0

